{
 "metadata": {
  "name": ""
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#encoding=utf8\n",
      "import pymongo\n",
      "import simhash \n",
      "\n",
      "Article = pymongo.Connection().stock.Article\n",
      "\n",
      "\n",
      "def pr(t):\n",
      "    print json.dumps(t, ensure_ascii=False,indent=2)\n",
      "id=0\n",
      "bu = simhash.SimhashIndex({},k=50)\n",
      "for i in Article.find({},{\"_id\":1,\"sim\":1}).limit(10000):\n",
      "    obj_id = i['_id']\n",
      "    hash_int = simhash.Simhash(int(i['sim']))\n",
      "    bu.add( obj_id,hash_int)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "bu.k=13\n",
      "for item in Article.find().limit(5):\n",
      "    obj_id = item['_id']\n",
      "    sim = simhash.Simhash(int(item['sim']))\n",
      "#     print sim,\n",
      "    articles = bu.get_near_dups(sim)\n",
      "    print item['title']\n",
      "    for i in articles:\n",
      "#         print i\n",
      "        print \"##\",Article.find_one({\"_id\":bson.objectid.ObjectId(i)})['title']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(5/15)\u5341\u5927\u535a\u6587\u7cbe\u7cb9\uff1a\u4e09\u8fde\u9634\u540e\u5c06\u5982\u4f55\u6f14\u53d8\uff1f\n",
        "## \u201c\u7164\u98de\u8272\u821e\u201d\u96be\u6301\u7eed \u954d\u4ef7\u4ecd\u6709\u51b2\u9ad8\u673a\u4f1a\n",
        "## \u201c\u7164\u98de\u8272\u821e\u201d\u96be\u6301\u7eed \u954d\u4ef7\u4ecd\u6709\u51b2\u9ad8\u673a\u4f1a\n",
        "## \u5341\u5927\u673a\u6784\u9884\u6d4b\u5927\u52bf\uff1a\u5411\u4e0b\u8003\u9a8c2000\u70b9\u652f\u6491\n",
        "## \u66dd\u5b8b\u536b\u5e73\u9000\u51fa\u7eff\u57ce\u4e2d\u56fd \u9a6c\u4e91\u4e2a\u4eba\u8eab\u4efd\u6536\u8d2d\u7403\u961f\n",
        "## \u201c\u7164\u98de\u8272\u821e\u201d\u96be\u6301\u7eed \u954d\u4ef7\u4ecd\u6709\u51b2\u9ad8\u673a\u4f1a\n",
        "## (5/15)\u5341\u5927\u535a\u6587\u7cbe\u7cb9\uff1a\u4e09\u8fde\u9634\u540e\u5c06\u5982\u4f55\u6f14\u53d8\uff1f\n",
        "\u8010\u5fc3\u4f4e\u5438\u4e09\u7c7b\u80a1\n",
        "## \u5348\u540e\u5173\u6ce8\u80fd\u5426\u5411\u4e0a\u53cd\u62c9\n",
        "## \u8dcc\u8dcc\u4e0d\u4f11\u4f55\u65f6\u4e86\uff1f\n",
        "## \u8dcc\u8dcc\u4e0d\u4f11\u4f55\u65f6\u4e86\uff1f\n",
        "## \u8010\u5fc3\u4f4e\u5438\u4e09\u7c7b\u80a1\n",
        "## \u4e16\u754c\u6700\u5927\u84dd\u94bb2400\u4e07\u7f8e\u5143\u62cd\u51fa \u884c\u5bb6\u79f0\u662f\u6781\u54c1(\u56fe)\n",
        "## \u90e8\u5206\u4ea7\u54c1\u5df2\u80fd\u7528\u4e8e\u9875\u5ca9\u6c14\u5f00\u91c7 \u5c71\u4e1c\u58a8\u9f99\u5f3a\u52bf\u4e0a\u6da8\n",
        "## \u9707\u8361\u4e0b\u884c \u56db\u7c7b\u80a1\u6210A\u80a1\u201c\u7ede\u8089\u673a\u201d\n",
        "## \u5348\u540e\u5173\u6ce8\u80fd\u5426\u5411\u4e0a\u53cd\u62c9\n",
        "## \u9707\u8361\u4e0b\u884c \u56db\u7c7b\u80a1\u6210A\u80a1\u201c\u7ede\u8089\u673a\u201d\n",
        "## \u9707\u8361\u4e0b\u884c \u56db\u7c7b\u80a1\u6210A\u80a1\u201c\u7ede\u8089\u673a\u201d\n",
        "## \u8010\u5fc3\u4f4e\u5438\u4e09\u7c7b\u80a1\n",
        "## \u8010\u5fc3\u4f4e\u5438\u4e09\u7c7b\u80a1\n",
        "## \u5348\u540e\u5173\u6ce8\u80fd\u5426\u5411\u4e0a\u53cd\u62c9\n",
        "## \u9707\u8361\u4e0b\u884c \u56db\u7c7b\u80a1\u6210A\u80a1\u201c\u7ede\u8089\u673a\u201d\n",
        "## \u6caa\u6df1B\u6307\u53cc\u53cc\u6536\u8dcc \u6caa\u5e02B\u6307\u8dcc\u5e45\u4e3a0.32%\n",
        "\u4e2d\u56fd\u9a7b\u8d8a\u5357\u5927\u4f7f\u9986\u518d\u6b21\u63d0\u9192\u516c\u6c11\u6ce8\u610f\u4eba\u8eab\u8d22\u4ea7\u5b89\u5168\n",
        "## \u8fdb\u53e3\u539f\u6cb9\u4f7f\u7528\u6743\u7ec6\u5219\u51fa\u53f0\u5728\u5373 \u6709\u6d77\u5916\u8d44\u6e90\u516c\u53f8\u53d7\u76ca\n",
        "## \u4e2d\u56fd\u9a7b\u8d8a\u5357\u5927\u4f7f\u9986\u518d\u6b21\u63d0\u9192\u516c\u6c11\u6ce8\u610f\u4eba\u8eab\u8d22\u4ea7\u5b89\u5168\n",
        "## \u6caa\u6df1\u4e24\u5e02\u5468\u4e94\u5fae\u5e45\u4f4e\u5f00 \u5238\u5546\u80a1\u9886\u6da8\n",
        "## \u7f8e\u56fd5\u6708\u8d39\u57ce\u8054\u50a8\u5236\u9020\u4e1a\u6307\u6570\u964d\u81f315.4\n",
        "\u6210\u4ea4\u91cf\u840e\u7f29\u7591\u8bf1\u7a7a \u6295\u8d44\u8005\u9700\u8c28\u614e\n",
        "## \u6210\u4ea4\u91cf\u840e\u7f29\u7591\u8bf1\u7a7a \u6295\u8d44\u8005\u9700\u8c28\u614e\n",
        "\u5f20\u88d5B\u88ab\u5254\u9664MSCI\u4e2d\u56fd\u6307\u6570\n",
        "## \u8868\u73b0\u504f\u5f31 \u7b49\u5f85\u65b0\u673a\u4f1a\n",
        "## \u8868\u73b0\u504f\u5f31 \u7b49\u5f85\u65b0\u673a\u4f1a\n",
        "## \u8868\u73b0\u504f\u5f31 \u7b49\u5f85\u65b0\u673a\u4f1a\n"
       ]
      }
     ],
     "prompt_number": 27
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "Article.find_one({\"_id\":ObjectId('5374c1267f949f34902c46f1')})"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "NameError",
       "evalue": "name 'ObjectId' is not defined",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
        "\u001b[1;32m<ipython-input-16-e2a4741c92aa>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mArticle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_one\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m{\u001b[0m\u001b[1;34m\"_id\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mObjectId\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'5374c1267f949f34902c46f1'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
        "\u001b[1;31mNameError\u001b[0m: name 'ObjectId' is not defined"
       ]
      }
     ],
     "prompt_number": 16
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "type(Article.find_one()['_id'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 20,
       "text": [
        "bson.objectid.ObjectId"
       ]
      }
     ],
     "prompt_number": 20
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "bson.objectid.ObjectId('5374c1267f949f34902c46f1')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 25,
       "text": [
        "ObjectId('5374c1267f949f34902c46f1')"
       ]
      }
     ],
     "prompt_number": 25
    }
   ],
   "metadata": {}
  }
 ]
}